'''
@Author: ysz
@Date: 2019-12-26 10:17:22
@LastEditors  : ysz
@LastEditTime : 2020-01-10 11:51:09
@Description: 百姓网租房和二手房的公有部分
'''

import requests
import re
import random
import time
import datetime
from lxml import etree
import os
import os.path

from mysqlhelp import MysqlHelp
from get_user_agent import User_Agent_Help

myhelp=MysqlHelp()
my_useragent=User_Agent_Help()

class Baixing(object):


    #初始化方法,初始化url和header头部信息
    def __init__(self,url):
        self.url=url

    #获取所有的列表页，并遍历每一个列表页，然后获取该列表页中所有详情页的url
    def all_page_list(self):
        headers={
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
                'Accept-Encoding':'gzip, deflate, br',
                'Accept-Language':'zh-CN,zh;q=0.9',
                'Cache-Control': 'max-age=0',
                # 'Connection': 'keep-alive',
                'Cookie': 'suid=3807722599; __trackId=157708634659878; _ga=GA1.2.1771081173.1577086347; _gid=GA1.2.2007515902.1577086347; __admx_track_id=vfJpS1IJYFYMR3fbVh_ZGg; __admx_track_id.sig=Czr8QkwjeddFn8A7chX1oS7OPlQ; __s=atsjf95rrifs47m7eav25r1f93; _auth_redirect=https%3A%2F%2Fshenyang.baixing.com%2Fershoufang%2Fa2056054238.html%3Ffrom%3Dvip; seo_ad_test=1; __city=shenyang; Hm_lvt_5a727f1b4acc5725516637e03b07d3d2=1577167250,1577173005,1577173789,1577173801; __sense_session_pv=9; Hm_lpvt_5a727f1b4acc5725516637e03b07d3d2=1577173807',
                'Host': 'shenyang.baixing.com',
                'If-Modified-Since': 'Tue, 24 Dec 2019 07:50:05 GMT',
                'Referer': 'https://shenyang.baixing.com/',
                'Sec-Fetch-Mode': 'navigate',
                'Sec-Fetch-Site': 'same-origin',
                'Sec-Fetch-User': '?1',
                'Upgrade-Insecure-Requests': '1',
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
            }
        res=requests.get(self.url,headers=headers)
        print(res.status_code)
        # print(res.text)
        html_ele=etree.HTML(res.text)
        #获取列表页的最终值
        last_page_number=html_ele.xpath('//ul[@class="list-pagination"]/li[last()-1]/a/text()')[0]
        for page_number in range(int(last_page_number)):
            time.sleep(random.randint(50,60))
            page_url=self.url.format(page_number+1)
            print(page_url,'列表页的url')
            self.list_page(page_url)
            

    #--------从列表页中获取详情页的url，从而遍历获取该列表页的中的关于详情页中的相关信息
    def list_page(self,url):
        headers={
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
                'Accept-Encoding': 'gzip, deflate, br',
                'Accept-Language': 'zh-CN,zh;q=0.9',
                'Cache-Control': 'max-age=0',
                # 'Connection': 'keep-alive',
                'Cookie': 'suid=3807722599; __trackId=157708634659878; _ga=GA1.2.1771081173.1577086347; _gid=GA1.2.2007515902.1577086347; __admx_track_id=vfJpS1IJYFYMR3fbVh_ZGg; __admx_track_id.sig=Czr8QkwjeddFn8A7chX1oS7OPlQ; __s=atsjf95rrifs47m7eav25r1f93; _auth_redirect=https%3A%2F%2Fshenyang.baixing.com%2Fershoufang%2Fa2056054238.html%3Ffrom%3Dvip; seo_ad_test=1; __city=shenyang; Hm_lvt_5a727f1b4acc5725516637e03b07d3d2=1577167250,1577173005,1577173789,1577173801; __sense_session_pv=11; _gat=1; Hm_lpvt_5a727f1b4acc5725516637e03b07d3d2=1577174629',
                'Host': 'shenyang.baixing.com',
                'If-Modified-Since': 'Tue, 24 Dec 2019 08:03:46 GMT',
                'Referer': 'https://shenyang.baixing.com/ershoufang/?src=topbar',
                'Sec-Fetch-Mode': 'navigate',
                'Sec-Fetch-Site': 'same-origin',
                'Sec-Fetch-User': '?1',
                'Upgrade-Insecure-Requests': '1',
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
            }
        res=requests.get(url,headers=headers)
        response=res.text
        html_ele=etree.HTML(response)
        #获取详情页的url
        detail_page_urls=html_ele.xpath('//div[@class="media-body-title"]/a/@href')
        detail_page_urls=[ item for item in detail_page_urls if "shenyang.baixing.com" in item]
        print(len(detail_page_urls),'详情页的url的数量')
        for detail_page_url in detail_page_urls:
            time.sleep(random.randint(40,60))
            print(detail_page_url)
            self.get_detail_page(detail_page_url)
            

    #获取详情页信息
    def get_detail_page(self,url):
        assert(0)


    # #通过在详情页获取的图片url，下载图片到指定文件夹里
    # def down_img_pic(self,img_url,referer,house_type):
    #     print(house_type)
    #     headers={
    #             'Referer': referer,
    #             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
    #         }
    #     response=requests.get(img_url,headers=headers)
    #     #获取图片名字
    #     filename=img_url.split('/')[-1].split('.')[0]
    #     # tail_name=img_url.split('.')[-1].split('_')[0]

    #     #拼接字符串，组成图片名字
    #     filename=filename+'.jpg'
    #     dir_name=str(datetime.datetime.now()).split(' ')[0].replace('-','')
    #     if not os.path.exists(f'./百姓_二手房/download/{house_type}/'):
    #         os.mkdir(f'./百姓_二手房/download/{house_type}/')
    #     if not os.path.exists(f'./百姓_二手房/download/{house_type}/'+dir_name):
    #         os.mkdir(f'./百姓_二手房/download/{house_type}/{dir_name}/')
    #     with open(f'./百姓_二手房/download/{house_type}/{dir_name}/{filename}','wb') as fp:
    #         fp.write(response.content)
    #     pic_path='/百姓_二手房/download/'+house_type+'/'+dir_name+'/'+filename
    #     print(pic_path)
    #     return pic_path


    #模糊判断房源所在小区的楼层，并返回楼层数
    def judge_scope(self,floor,total_floor):
        medium=int(total_floor/3)
        if floor=='高层':
            floor=random.randrange(1+medium*2,total_floor+1)
        elif floor=='中层':
            floor=random.randrange(1+medium*1,1+medium*2+1)
        else:
            floor=random.randrange(1,1+medium*1+1)
        return floor


#程序的入口  
if __name__ == "__main__":
    path='./百姓_二手房/download'
    if not os.path.exists(path):
        os.mkdir(path)
    user_agent=my_useragent.get_useragent()
    url=''
    my=Baixing(url)
    my.all_page_list()

